﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using CommonFramework.Utility;
using System.Text.RegularExpressions;
namespace Xinkui.Spider.Spider
{
    public class SpiderProvider
    {
        /// <summary>
        /// 抓取根地址
        /// </summary>
        protected string LinkUrl = WebHelper.AppSettings["SpiderLink"];

        /// <summary>
        /// 商品分类地址
        /// </summary>
        protected int NavID = WebHelper.AppSettings["SpiderLink"].ToType<int>(0);

        protected string LinkHost = "http://www.alibaba.com/";

        protected string NextPage = "";

        /// <summary>
        /// 当前页
        /// </summary>
        protected int PageIndex = 1;

        /// <summary>
        /// 总页数
        /// </summary>
        protected int AllPageCount = 1;

        /// <summary>
        /// 列表开始项
        /// </summary>
        protected string strListStart = @"<div class=""f-icon m-item";

        /// <summary>
        /// 列表结束项
        /// </summary>
        protected string strListEnd = "Go to Page";

        public SpiderProvider()
        {
            NextPage = LinkUrl;



            do
            {
                //抓取大分类
                string listPage = HttpUtil.PostPage(NextPage, null);
                listPage = listPage.Substring(listPage.IndexOf(strListStart));
                listPage = listPage.Substring(0, listPage.IndexOf(strListEnd));

                //当前页内容

                Regex regLinkContext = new Regex(@"<h2(.*?)>([^<]*?)(<.*?>([^<>]*)</.*?>)([^<]*?)</h2>", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Compiled | RegexOptions.CultureInvariant);
                Regex regLink = new Regex(@"<a.*?href=[""']*([^""']+)[""']*""[^>]*>(.*?)<\/a>");
                foreach (Group company in regLinkContext.Match(listPage).Groups)
                {
                    Match companyhref = regLink.Match(company.Value);
                    string companyLink = companyhref.Groups[1].Value;
                    string companyName = companyhref.Groups[2].Value;
                    CompanySpider spider = new CompanySpider(companyLink, companyName);
                    break;
                }


                //下一页地址
                NextPage = string.Empty;
                Regex regNext = new Regex(@"<a class=""next""\shref *= *['""]*(\S+)[""']\s.*><i></i></a>", RegexOptions.IgnoreCase);
                if (regNext.IsMatch(listPage))
                {
                    NextPage = regNext.Match(listPage).Groups[1].Value;
                    if (!string.IsNullOrEmpty(NextPage))
                    {
                        NextPage = LinkHost + NextPage;
                    }
                }
                break;

            } while (!string.IsNullOrEmpty(NextPage));



        }

    }
}
